{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anomalus Vertices Detection in Twitter Graph\n",
    "In this notebook I am going to demonstrate how I use the method described in https://arxiv.org/abs/1610.07525 to detect anomlies in Twitter's social network.\n",
    "\n",
    "To install the packge please read the [installion intruction](https://github.com/Kagandi/anomalous-vertices-detection)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from anomalous_vertices_detection.graph_learning_controller import GraphLearningController\n",
    "from anomalous_vertices_detection.learners.sklearner import SkLearner\n",
    "from anomalous_vertices_detection.datasets.twitter import load_data\n",
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The basic installation install the packge with networkx as its graph analysis package, it is also possible to install and use iGraph"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we will define what considered as positive and negative labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = {\"neg\": \"Real\", \"pos\": \"Fake\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next we will load the graph of Twitter.\n",
    "load_data will return a graph object(twitter_graph) and a config object(twitter_config)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading graph...\n",
      "Data loaded.\n",
      "Generating 77017 vertices.\n",
      "77017 fake users generated.\n",
      "847191\n"
     ]
    }
   ],
   "source": [
    "twitter_graph, twitter_config = load_data(labels_map=labels)\n",
    "print(len(twiiter_graph.vertices))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The reuslt will be written in result_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_folder = \"../output/twitter\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some of the extracted feature can be usefull for understanding the result but they must not be used in the ml proccess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if twitter_graph.is_directed:\n",
    "    meta_data_cols = [\"dst\", \"src\", \"out_degree_v\", \"in_degree_v\", \"out_degree_u\", \"in_degree_u\"]\n",
    "else:\n",
    "    meta_data_cols = [\"dst\", \"src\", \"number_of_friends_u\", \"number_of_friends_v\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally we are goigng to execute the classification algorithm.\n",
    "The Twitter dataset is incomplete and missing a lot of data.\n",
    "In order to deal with this problem we are runnig the machine learning proccess 10 times and aggregating the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'glc' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-a662f2d241a1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m glc.classify_by_links(academia_graph, result_path, test_size={\"neg\": 1000, \"pos\": 100},\n\u001b[0m\u001b[0;32m      2\u001b[0m                       train_size={\"neg\": 20000, \"pos\": 20000}, meta_data_cols=meta_data_cols)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'glc' is not defined"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    twitter_config._name = \"twitter_\" + str(i)\n",
    "    glc = GraphLearningController(SkLearner(labels=labels), twitter_config)\n",
    "    result_path = os.path.join(output_folder, twitter_config.name  + \"_res.csv\")\n",
    "    glc.classify_by_links(twiiter_graph, result_path, test_size={\"neg\": 10000, \"pos\": 1000},\n",
    "                          train_size={\"neg\": 20000, \"pos\": 20000}, meta_data_cols=meta_data_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggreagate_res(data_folder, res_path):\n",
    "    results_frame = pd.DataFrame()\n",
    "    for f in os.listdir(data_folder):\n",
    "        temp_df = pd.read_csv(data_folder + \"\\\\\" + f,index_col=0)\n",
    "        results_frame = results_frame.append(temp_df)\n",
    "    results_frame = results_frame.groupby(\"src_id\").mean()\n",
    "\n",
    "    return results_frame.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = aggreagate_res(output_folder, \"res.csv\").sort_values(\"mean_link_label\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src_id</th>\n",
       "      <th>neg probability</th>\n",
       "      <th>edge number</th>\n",
       "      <th>pos probability</th>\n",
       "      <th>mean_link_label</th>\n",
       "      <th>sum_link_label</th>\n",
       "      <th>predicted_label</th>\n",
       "      <th>id</th>\n",
       "      <th>actual</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3595</th>\n",
       "      <td>2104351</td>\n",
       "      <td>0.037838</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.962162</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2104351.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3282</th>\n",
       "      <td>955631</td>\n",
       "      <td>0.044064</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.955936</td>\n",
       "      <td>0.981481</td>\n",
       "      <td>17.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>955631.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>352725</td>\n",
       "      <td>0.206864</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.793136</td>\n",
       "      <td>0.928205</td>\n",
       "      <td>12.066667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>352725.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>1455795</td>\n",
       "      <td>0.262503</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.737497</td>\n",
       "      <td>0.858333</td>\n",
       "      <td>20.600000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1455795.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>17643</td>\n",
       "      <td>0.189451</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.810549</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>13.200000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>17643.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2874</th>\n",
       "      <td>615155</td>\n",
       "      <td>0.287905</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.712095</td>\n",
       "      <td>0.794444</td>\n",
       "      <td>9.533333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>615155.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>5594</td>\n",
       "      <td>0.385141</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.614859</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>5594.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3564</th>\n",
       "      <td>1886070</td>\n",
       "      <td>0.237128</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.762872</td>\n",
       "      <td>0.779487</td>\n",
       "      <td>10.133333</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1886070.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3485</th>\n",
       "      <td>1444212</td>\n",
       "      <td>0.234757</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.765243</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>16.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1444212.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1956</th>\n",
       "      <td>262232</td>\n",
       "      <td>0.255232</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.744768</td>\n",
       "      <td>0.761404</td>\n",
       "      <td>14.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>262232.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1916</th>\n",
       "      <td>248965</td>\n",
       "      <td>0.263690</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.736310</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>14.600000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>248965.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>456</td>\n",
       "      <td>0.379521</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.620479</td>\n",
       "      <td>0.707937</td>\n",
       "      <td>14.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1770</th>\n",
       "      <td>215624</td>\n",
       "      <td>0.356370</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.643630</td>\n",
       "      <td>0.697778</td>\n",
       "      <td>10.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>215624.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2666</th>\n",
       "      <td>496941</td>\n",
       "      <td>0.405250</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.594750</td>\n",
       "      <td>0.693333</td>\n",
       "      <td>10.400000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>496941.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1665</th>\n",
       "      <td>185732</td>\n",
       "      <td>0.335138</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.664862</td>\n",
       "      <td>0.686111</td>\n",
       "      <td>16.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>185732.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>23739</td>\n",
       "      <td>0.390818</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.609182</td>\n",
       "      <td>0.678788</td>\n",
       "      <td>29.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>23739.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2665</th>\n",
       "      <td>496919</td>\n",
       "      <td>0.391924</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.608076</td>\n",
       "      <td>0.675000</td>\n",
       "      <td>10.800000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>496919.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>23907</td>\n",
       "      <td>0.380953</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.619047</td>\n",
       "      <td>0.648718</td>\n",
       "      <td>16.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>23907.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>42133</td>\n",
       "      <td>0.437460</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.562540</td>\n",
       "      <td>0.597333</td>\n",
       "      <td>14.933333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>42133.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>416873</td>\n",
       "      <td>0.378616</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.621384</td>\n",
       "      <td>0.587879</td>\n",
       "      <td>6.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>416873.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3223</th>\n",
       "      <td>878437</td>\n",
       "      <td>0.360188</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.639812</td>\n",
       "      <td>0.587500</td>\n",
       "      <td>9.400000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>878437.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>880719</td>\n",
       "      <td>0.422965</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.577035</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>9.333333</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>880719.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2664</th>\n",
       "      <td>496918</td>\n",
       "      <td>0.463268</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.536732</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>6.933333</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>496918.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3031</th>\n",
       "      <td>727279</td>\n",
       "      <td>0.455200</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.544800</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>6.933333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>727279.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2805</th>\n",
       "      <td>568345</td>\n",
       "      <td>0.449091</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.550909</td>\n",
       "      <td>0.576471</td>\n",
       "      <td>9.800000</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>568345.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1126</th>\n",
       "      <td>113506</td>\n",
       "      <td>0.436098</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.563902</td>\n",
       "      <td>0.570139</td>\n",
       "      <td>54.733333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>113506.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947</th>\n",
       "      <td>668723</td>\n",
       "      <td>0.469345</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.530655</td>\n",
       "      <td>0.568182</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>668723.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>6402</td>\n",
       "      <td>0.444703</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.555297</td>\n",
       "      <td>0.551515</td>\n",
       "      <td>6.066667</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>6402.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1769</th>\n",
       "      <td>215410</td>\n",
       "      <td>0.457158</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.542842</td>\n",
       "      <td>0.544444</td>\n",
       "      <td>6.533333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>215410.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3502</th>\n",
       "      <td>1560662</td>\n",
       "      <td>0.468981</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.531019</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>17.333333</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>1560662.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1718</th>\n",
       "      <td>200549</td>\n",
       "      <td>0.991372</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.008628</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200549.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1717</th>\n",
       "      <td>200465</td>\n",
       "      <td>0.994578</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.005422</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200465.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1716</th>\n",
       "      <td>200270</td>\n",
       "      <td>0.986267</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.013733</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200270.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1715</th>\n",
       "      <td>200194</td>\n",
       "      <td>0.998238</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.001762</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200194.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1714</th>\n",
       "      <td>200146</td>\n",
       "      <td>0.990160</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.009840</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200146.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1713</th>\n",
       "      <td>199026</td>\n",
       "      <td>0.990207</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.009793</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>199026.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>196830</td>\n",
       "      <td>0.981306</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.018694</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>196830.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1707</th>\n",
       "      <td>196384</td>\n",
       "      <td>0.983809</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.016191</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>196384.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700</th>\n",
       "      <td>194867</td>\n",
       "      <td>0.963333</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.036667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194867.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1699</th>\n",
       "      <td>194766</td>\n",
       "      <td>0.975424</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.024576</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194766.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1697</th>\n",
       "      <td>194634</td>\n",
       "      <td>0.987829</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.012171</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194634.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696</th>\n",
       "      <td>194633</td>\n",
       "      <td>0.982690</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.017310</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194633.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1692</th>\n",
       "      <td>194097</td>\n",
       "      <td>0.975205</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.024795</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194097.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1688</th>\n",
       "      <td>192525</td>\n",
       "      <td>0.982586</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0.017414</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>192525.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1681</th>\n",
       "      <td>191104</td>\n",
       "      <td>0.982146</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.017854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>191104.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1670</th>\n",
       "      <td>187088</td>\n",
       "      <td>0.990946</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.009054</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>187088.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1669</th>\n",
       "      <td>186716</td>\n",
       "      <td>0.989697</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.010303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>186716.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1667</th>\n",
       "      <td>186282</td>\n",
       "      <td>0.968713</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.031287</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>186282.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>185057</td>\n",
       "      <td>0.971470</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.028530</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>185057.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1659</th>\n",
       "      <td>184540</td>\n",
       "      <td>0.989574</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.010426</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184540.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1658</th>\n",
       "      <td>184538</td>\n",
       "      <td>0.991623</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.008377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184538.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1654</th>\n",
       "      <td>184427</td>\n",
       "      <td>0.964500</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.035500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184427.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653</th>\n",
       "      <td>184404</td>\n",
       "      <td>0.987467</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.012533</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184404.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1652</th>\n",
       "      <td>184403</td>\n",
       "      <td>0.986272</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.013728</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184403.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650</th>\n",
       "      <td>184375</td>\n",
       "      <td>0.976997</td>\n",
       "      <td>83.0</td>\n",
       "      <td>0.023003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184375.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>184355</td>\n",
       "      <td>0.984859</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0.015141</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184355.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1646</th>\n",
       "      <td>184274</td>\n",
       "      <td>0.978667</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.021333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184274.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>184070</td>\n",
       "      <td>0.976638</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.023362</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184070.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1633</th>\n",
       "      <td>179854</td>\n",
       "      <td>0.989791</td>\n",
       "      <td>77.0</td>\n",
       "      <td>0.010209</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>179854.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3615</th>\n",
       "      <td>2351897</td>\n",
       "      <td>0.983939</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.016061</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2351897.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3616 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       src_id  neg probability  edge number  pos probability  mean_link_label  \\\n",
       "3595  2104351         0.037838         13.0         0.962162         1.000000   \n",
       "3282   955631         0.044064         18.0         0.955936         0.981481   \n",
       "2289   352725         0.206864         13.0         0.793136         0.928205   \n",
       "3488  1455795         0.262503         24.0         0.737497         0.858333   \n",
       "293     17643         0.189451         16.0         0.810549         0.825000   \n",
       "2874   615155         0.287905         12.0         0.712095         0.794444   \n",
       "62       5594         0.385141         14.0         0.614859         0.785714   \n",
       "3564  1886070         0.237128         13.0         0.762872         0.779487   \n",
       "3485  1444212         0.234757         21.0         0.765243         0.777778   \n",
       "1956   262232         0.255232         19.0         0.744768         0.761404   \n",
       "1916   248965         0.263690         20.0         0.736310         0.730000   \n",
       "25        456         0.379521         21.0         0.620479         0.707937   \n",
       "1770   215624         0.356370         15.0         0.643630         0.697778   \n",
       "2666   496941         0.405250         15.0         0.594750         0.693333   \n",
       "1665   185732         0.335138         24.0         0.664862         0.686111   \n",
       "435     23739         0.390818         44.0         0.609182         0.678788   \n",
       "2665   496919         0.391924         16.0         0.608076         0.675000   \n",
       "449     23907         0.380953         26.0         0.619047         0.648718   \n",
       "617     42133         0.437460         25.0         0.562540         0.597333   \n",
       "2485   416873         0.378616         11.0         0.621384         0.587879   \n",
       "3223   878437         0.360188         16.0         0.639812         0.587500   \n",
       "3230   880719         0.422965         16.0         0.577035         0.583333   \n",
       "2664   496918         0.463268         12.0         0.536732         0.577778   \n",
       "3031   727279         0.455200         12.0         0.544800         0.577778   \n",
       "2805   568345         0.449091         17.0         0.550909         0.576471   \n",
       "1126   113506         0.436098         96.0         0.563902         0.570139   \n",
       "2947   668723         0.469345         44.0         0.530655         0.568182   \n",
       "103      6402         0.444703         11.0         0.555297         0.551515   \n",
       "1769   215410         0.457158         12.0         0.542842         0.544444   \n",
       "3502  1560662         0.468981         32.0         0.531019         0.541667   \n",
       "...       ...              ...          ...              ...              ...   \n",
       "1718   200549         0.991372         33.0         0.008628         0.000000   \n",
       "1717   200465         0.994578         15.0         0.005422         0.000000   \n",
       "1716   200270         0.986267         25.0         0.013733         0.000000   \n",
       "1715   200194         0.998238         14.0         0.001762         0.000000   \n",
       "1714   200146         0.990160         25.0         0.009840         0.000000   \n",
       "1713   199026         0.990207         29.0         0.009793         0.000000   \n",
       "1709   196830         0.981306         36.0         0.018694         0.000000   \n",
       "1707   196384         0.983809         32.0         0.016191         0.000000   \n",
       "1700   194867         0.963333         12.0         0.036667         0.000000   \n",
       "1699   194766         0.975424         60.0         0.024576         0.000000   \n",
       "1697   194634         0.987829         41.0         0.012171         0.000000   \n",
       "1696   194633         0.982690         29.0         0.017310         0.000000   \n",
       "1692   194097         0.975205         36.0         0.024795         0.000000   \n",
       "1688   192525         0.982586         63.0         0.017414         0.000000   \n",
       "1681   191104         0.982146         32.0         0.017854         0.000000   \n",
       "1670   187088         0.990946         31.0         0.009054         0.000000   \n",
       "1669   186716         0.989697         11.0         0.010303         0.000000   \n",
       "1667   186282         0.968713         14.0         0.031287         0.000000   \n",
       "1664   185057         0.971470         31.0         0.028530         0.000000   \n",
       "1659   184540         0.989574         71.0         0.010426         0.000000   \n",
       "1658   184538         0.991623         46.0         0.008377         0.000000   \n",
       "1654   184427         0.964500         18.0         0.035500         0.000000   \n",
       "1653   184404         0.987467         20.0         0.012533         0.000000   \n",
       "1652   184403         0.986272         27.0         0.013728         0.000000   \n",
       "1650   184375         0.976997         83.0         0.023003         0.000000   \n",
       "1648   184355         0.984859         59.0         0.015141         0.000000   \n",
       "1646   184274         0.978667         14.0         0.021333         0.000000   \n",
       "1641   184070         0.976638         21.0         0.023362         0.000000   \n",
       "1633   179854         0.989791         77.0         0.010209         0.000000   \n",
       "3615  2351897         0.983939         11.0         0.016061         0.000000   \n",
       "\n",
       "      sum_link_label  predicted_label         id  actual  \n",
       "3595       13.000000         1.000000  2104351.0     1.0  \n",
       "3282       17.666667         1.000000   955631.0     0.0  \n",
       "2289       12.066667         1.000000   352725.0     1.0  \n",
       "3488       20.600000         1.000000  1455795.0     1.0  \n",
       "293        13.200000         1.000000    17643.0     0.0  \n",
       "2874        9.533333         1.000000   615155.0     0.0  \n",
       "62         11.000000         0.866667     5594.0     0.0  \n",
       "3564       10.133333         0.933333  1886070.0     0.0  \n",
       "3485       16.333333         1.000000  1444212.0     1.0  \n",
       "1956       14.466667         1.000000   262232.0     1.0  \n",
       "1916       14.600000         1.000000   248965.0     0.0  \n",
       "25         14.866667         1.000000      456.0     0.0  \n",
       "1770       10.466667         1.000000   215624.0     1.0  \n",
       "2666       10.400000         0.866667   496941.0     1.0  \n",
       "1665       16.466667         1.000000   185732.0     0.0  \n",
       "435        29.866667         1.000000    23739.0     1.0  \n",
       "2665       10.800000         0.800000   496919.0     1.0  \n",
       "449        16.866667         1.000000    23907.0     1.0  \n",
       "617        14.933333         0.733333    42133.0     0.0  \n",
       "2485        6.466667         1.000000   416873.0     0.0  \n",
       "3223        9.400000         1.000000   878437.0     0.0  \n",
       "3230        9.333333         0.800000   880719.0     1.0  \n",
       "2664        6.933333         0.533333   496918.0     1.0  \n",
       "3031        6.933333         0.733333   727279.0     0.0  \n",
       "2805        9.800000         0.733333   568345.0     0.0  \n",
       "1126       54.733333         0.733333   113506.0     0.0  \n",
       "2947       25.000000         0.600000   668723.0     0.0  \n",
       "103         6.066667         0.933333     6402.0     0.0  \n",
       "1769        6.533333         0.733333   215410.0     1.0  \n",
       "3502       17.333333         0.600000  1560662.0     0.0  \n",
       "...              ...              ...        ...     ...  \n",
       "1718        0.000000         0.000000   200549.0     0.0  \n",
       "1717        0.000000         0.000000   200465.0     0.0  \n",
       "1716        0.000000         0.000000   200270.0     0.0  \n",
       "1715        0.000000         0.000000   200194.0     0.0  \n",
       "1714        0.000000         0.000000   200146.0     0.0  \n",
       "1713        0.000000         0.000000   199026.0     0.0  \n",
       "1709        0.000000         0.000000   196830.0     0.0  \n",
       "1707        0.000000         0.000000   196384.0     0.0  \n",
       "1700        0.000000         0.000000   194867.0     0.0  \n",
       "1699        0.000000         0.000000   194766.0     0.0  \n",
       "1697        0.000000         0.000000   194634.0     0.0  \n",
       "1696        0.000000         0.000000   194633.0     0.0  \n",
       "1692        0.000000         0.000000   194097.0     0.0  \n",
       "1688        0.000000         0.000000   192525.0     0.0  \n",
       "1681        0.000000         0.000000   191104.0     0.0  \n",
       "1670        0.000000         0.000000   187088.0     0.0  \n",
       "1669        0.000000         0.000000   186716.0     1.0  \n",
       "1667        0.000000         0.000000   186282.0     0.0  \n",
       "1664        0.000000         0.000000   185057.0     0.0  \n",
       "1659        0.000000         0.000000   184540.0     0.0  \n",
       "1658        0.000000         0.000000   184538.0     0.0  \n",
       "1654        0.000000         0.000000   184427.0     0.0  \n",
       "1653        0.000000         0.000000   184404.0     0.0  \n",
       "1652        0.000000         0.000000   184403.0     1.0  \n",
       "1650        0.000000         0.000000   184375.0     0.0  \n",
       "1648        0.000000         0.000000   184355.0     0.0  \n",
       "1646        0.000000         0.000000   184274.0     0.0  \n",
       "1641        0.000000         0.000000   184070.0     0.0  \n",
       "1633        0.000000         0.000000   179854.0     0.0  \n",
       "3615        0.000000         0.000000  2351897.0     0.0  \n",
       "\n",
       "[3616 rows x 9 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to calculate precision at k (How many releveant results at k) we will have to added two additional columns:\n",
    "1. The sum of the positive examples.\n",
    "2. k  the total number of resutls until the specific row. (row number)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"actual_sum\"] = df[\"actual\"].cumsum()\n",
    "df[\"k\"] = 1\n",
    "df[\"k\"] = df[\"k\"].cumsum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>src_id</th>\n",
       "      <th>neg probability</th>\n",
       "      <th>edge number</th>\n",
       "      <th>pos probability</th>\n",
       "      <th>mean_link_label</th>\n",
       "      <th>sum_link_label</th>\n",
       "      <th>predicted_label</th>\n",
       "      <th>id</th>\n",
       "      <th>actual</th>\n",
       "      <th>a_sum</th>\n",
       "      <th>k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3595</th>\n",
       "      <td>2104351</td>\n",
       "      <td>0.037838</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.962162</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2104351.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3282</th>\n",
       "      <td>955631</td>\n",
       "      <td>0.044064</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.955936</td>\n",
       "      <td>0.981481</td>\n",
       "      <td>17.666667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>955631.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>352725</td>\n",
       "      <td>0.206864</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.793136</td>\n",
       "      <td>0.928205</td>\n",
       "      <td>12.066667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>352725.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>1455795</td>\n",
       "      <td>0.262503</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.737497</td>\n",
       "      <td>0.858333</td>\n",
       "      <td>20.600000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1455795.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>17643</td>\n",
       "      <td>0.189451</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.810549</td>\n",
       "      <td>0.825000</td>\n",
       "      <td>13.200000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>17643.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2874</th>\n",
       "      <td>615155</td>\n",
       "      <td>0.287905</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.712095</td>\n",
       "      <td>0.794444</td>\n",
       "      <td>9.533333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>615155.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>5594</td>\n",
       "      <td>0.385141</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.614859</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>5594.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3564</th>\n",
       "      <td>1886070</td>\n",
       "      <td>0.237128</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.762872</td>\n",
       "      <td>0.779487</td>\n",
       "      <td>10.133333</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>1886070.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3485</th>\n",
       "      <td>1444212</td>\n",
       "      <td>0.234757</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.765243</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>16.333333</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1444212.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1956</th>\n",
       "      <td>262232</td>\n",
       "      <td>0.255232</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.744768</td>\n",
       "      <td>0.761404</td>\n",
       "      <td>14.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>262232.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1916</th>\n",
       "      <td>248965</td>\n",
       "      <td>0.263690</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.736310</td>\n",
       "      <td>0.730000</td>\n",
       "      <td>14.600000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>248965.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>456</td>\n",
       "      <td>0.379521</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.620479</td>\n",
       "      <td>0.707937</td>\n",
       "      <td>14.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>456.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1770</th>\n",
       "      <td>215624</td>\n",
       "      <td>0.356370</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.643630</td>\n",
       "      <td>0.697778</td>\n",
       "      <td>10.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>215624.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2666</th>\n",
       "      <td>496941</td>\n",
       "      <td>0.405250</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.594750</td>\n",
       "      <td>0.693333</td>\n",
       "      <td>10.400000</td>\n",
       "      <td>0.866667</td>\n",
       "      <td>496941.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1665</th>\n",
       "      <td>185732</td>\n",
       "      <td>0.335138</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.664862</td>\n",
       "      <td>0.686111</td>\n",
       "      <td>16.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>185732.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>23739</td>\n",
       "      <td>0.390818</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.609182</td>\n",
       "      <td>0.678788</td>\n",
       "      <td>29.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>23739.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2665</th>\n",
       "      <td>496919</td>\n",
       "      <td>0.391924</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.608076</td>\n",
       "      <td>0.675000</td>\n",
       "      <td>10.800000</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>496919.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>23907</td>\n",
       "      <td>0.380953</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0.619047</td>\n",
       "      <td>0.648718</td>\n",
       "      <td>16.866667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>23907.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>42133</td>\n",
       "      <td>0.437460</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.562540</td>\n",
       "      <td>0.597333</td>\n",
       "      <td>14.933333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>42133.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>416873</td>\n",
       "      <td>0.378616</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.621384</td>\n",
       "      <td>0.587879</td>\n",
       "      <td>6.466667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>416873.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3223</th>\n",
       "      <td>878437</td>\n",
       "      <td>0.360188</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.639812</td>\n",
       "      <td>0.587500</td>\n",
       "      <td>9.400000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>878437.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>880719</td>\n",
       "      <td>0.422965</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.577035</td>\n",
       "      <td>0.583333</td>\n",
       "      <td>9.333333</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>880719.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2664</th>\n",
       "      <td>496918</td>\n",
       "      <td>0.463268</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.536732</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>6.933333</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>496918.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3031</th>\n",
       "      <td>727279</td>\n",
       "      <td>0.455200</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.544800</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>6.933333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>727279.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2805</th>\n",
       "      <td>568345</td>\n",
       "      <td>0.449091</td>\n",
       "      <td>17.0</td>\n",
       "      <td>0.550909</td>\n",
       "      <td>0.576471</td>\n",
       "      <td>9.800000</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>568345.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1126</th>\n",
       "      <td>113506</td>\n",
       "      <td>0.436098</td>\n",
       "      <td>96.0</td>\n",
       "      <td>0.563902</td>\n",
       "      <td>0.570139</td>\n",
       "      <td>54.733333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>113506.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947</th>\n",
       "      <td>668723</td>\n",
       "      <td>0.469345</td>\n",
       "      <td>44.0</td>\n",
       "      <td>0.530655</td>\n",
       "      <td>0.568182</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>668723.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>6402</td>\n",
       "      <td>0.444703</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.555297</td>\n",
       "      <td>0.551515</td>\n",
       "      <td>6.066667</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>6402.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1769</th>\n",
       "      <td>215410</td>\n",
       "      <td>0.457158</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.542842</td>\n",
       "      <td>0.544444</td>\n",
       "      <td>6.533333</td>\n",
       "      <td>0.733333</td>\n",
       "      <td>215410.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3502</th>\n",
       "      <td>1560662</td>\n",
       "      <td>0.468981</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.531019</td>\n",
       "      <td>0.541667</td>\n",
       "      <td>17.333333</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>1560662.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1718</th>\n",
       "      <td>200549</td>\n",
       "      <td>0.991372</td>\n",
       "      <td>33.0</td>\n",
       "      <td>0.008628</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200549.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1717</th>\n",
       "      <td>200465</td>\n",
       "      <td>0.994578</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.005422</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200465.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1716</th>\n",
       "      <td>200270</td>\n",
       "      <td>0.986267</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.013733</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200270.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1715</th>\n",
       "      <td>200194</td>\n",
       "      <td>0.998238</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.001762</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200194.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1714</th>\n",
       "      <td>200146</td>\n",
       "      <td>0.990160</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.009840</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>200146.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1713</th>\n",
       "      <td>199026</td>\n",
       "      <td>0.990207</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.009793</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>199026.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>196830</td>\n",
       "      <td>0.981306</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.018694</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>196830.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1707</th>\n",
       "      <td>196384</td>\n",
       "      <td>0.983809</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.016191</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>196384.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700</th>\n",
       "      <td>194867</td>\n",
       "      <td>0.963333</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.036667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194867.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1699</th>\n",
       "      <td>194766</td>\n",
       "      <td>0.975424</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.024576</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194766.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1697</th>\n",
       "      <td>194634</td>\n",
       "      <td>0.987829</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.012171</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194634.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696</th>\n",
       "      <td>194633</td>\n",
       "      <td>0.982690</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.017310</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194633.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1692</th>\n",
       "      <td>194097</td>\n",
       "      <td>0.975205</td>\n",
       "      <td>36.0</td>\n",
       "      <td>0.024795</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>194097.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1688</th>\n",
       "      <td>192525</td>\n",
       "      <td>0.982586</td>\n",
       "      <td>63.0</td>\n",
       "      <td>0.017414</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>192525.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1681</th>\n",
       "      <td>191104</td>\n",
       "      <td>0.982146</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.017854</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>191104.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1670</th>\n",
       "      <td>187088</td>\n",
       "      <td>0.990946</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.009054</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>187088.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>226.0</td>\n",
       "      <td>3602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1669</th>\n",
       "      <td>186716</td>\n",
       "      <td>0.989697</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.010303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>186716.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1667</th>\n",
       "      <td>186282</td>\n",
       "      <td>0.968713</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.031287</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>186282.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>185057</td>\n",
       "      <td>0.971470</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.028530</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>185057.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1659</th>\n",
       "      <td>184540</td>\n",
       "      <td>0.989574</td>\n",
       "      <td>71.0</td>\n",
       "      <td>0.010426</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184540.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1658</th>\n",
       "      <td>184538</td>\n",
       "      <td>0.991623</td>\n",
       "      <td>46.0</td>\n",
       "      <td>0.008377</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184538.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1654</th>\n",
       "      <td>184427</td>\n",
       "      <td>0.964500</td>\n",
       "      <td>18.0</td>\n",
       "      <td>0.035500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184427.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653</th>\n",
       "      <td>184404</td>\n",
       "      <td>0.987467</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.012533</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184404.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>227.0</td>\n",
       "      <td>3609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1652</th>\n",
       "      <td>184403</td>\n",
       "      <td>0.986272</td>\n",
       "      <td>27.0</td>\n",
       "      <td>0.013728</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184403.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650</th>\n",
       "      <td>184375</td>\n",
       "      <td>0.976997</td>\n",
       "      <td>83.0</td>\n",
       "      <td>0.023003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184375.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>184355</td>\n",
       "      <td>0.984859</td>\n",
       "      <td>59.0</td>\n",
       "      <td>0.015141</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184355.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1646</th>\n",
       "      <td>184274</td>\n",
       "      <td>0.978667</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.021333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184274.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>184070</td>\n",
       "      <td>0.976638</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.023362</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>184070.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1633</th>\n",
       "      <td>179854</td>\n",
       "      <td>0.989791</td>\n",
       "      <td>77.0</td>\n",
       "      <td>0.010209</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>179854.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3615</th>\n",
       "      <td>2351897</td>\n",
       "      <td>0.983939</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.016061</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2351897.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>3616</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3616 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       src_id  neg probability  edge number  pos probability  mean_link_label  \\\n",
       "3595  2104351         0.037838         13.0         0.962162         1.000000   \n",
       "3282   955631         0.044064         18.0         0.955936         0.981481   \n",
       "2289   352725         0.206864         13.0         0.793136         0.928205   \n",
       "3488  1455795         0.262503         24.0         0.737497         0.858333   \n",
       "293     17643         0.189451         16.0         0.810549         0.825000   \n",
       "2874   615155         0.287905         12.0         0.712095         0.794444   \n",
       "62       5594         0.385141         14.0         0.614859         0.785714   \n",
       "3564  1886070         0.237128         13.0         0.762872         0.779487   \n",
       "3485  1444212         0.234757         21.0         0.765243         0.777778   \n",
       "1956   262232         0.255232         19.0         0.744768         0.761404   \n",
       "1916   248965         0.263690         20.0         0.736310         0.730000   \n",
       "25        456         0.379521         21.0         0.620479         0.707937   \n",
       "1770   215624         0.356370         15.0         0.643630         0.697778   \n",
       "2666   496941         0.405250         15.0         0.594750         0.693333   \n",
       "1665   185732         0.335138         24.0         0.664862         0.686111   \n",
       "435     23739         0.390818         44.0         0.609182         0.678788   \n",
       "2665   496919         0.391924         16.0         0.608076         0.675000   \n",
       "449     23907         0.380953         26.0         0.619047         0.648718   \n",
       "617     42133         0.437460         25.0         0.562540         0.597333   \n",
       "2485   416873         0.378616         11.0         0.621384         0.587879   \n",
       "3223   878437         0.360188         16.0         0.639812         0.587500   \n",
       "3230   880719         0.422965         16.0         0.577035         0.583333   \n",
       "2664   496918         0.463268         12.0         0.536732         0.577778   \n",
       "3031   727279         0.455200         12.0         0.544800         0.577778   \n",
       "2805   568345         0.449091         17.0         0.550909         0.576471   \n",
       "1126   113506         0.436098         96.0         0.563902         0.570139   \n",
       "2947   668723         0.469345         44.0         0.530655         0.568182   \n",
       "103      6402         0.444703         11.0         0.555297         0.551515   \n",
       "1769   215410         0.457158         12.0         0.542842         0.544444   \n",
       "3502  1560662         0.468981         32.0         0.531019         0.541667   \n",
       "...       ...              ...          ...              ...              ...   \n",
       "1718   200549         0.991372         33.0         0.008628         0.000000   \n",
       "1717   200465         0.994578         15.0         0.005422         0.000000   \n",
       "1716   200270         0.986267         25.0         0.013733         0.000000   \n",
       "1715   200194         0.998238         14.0         0.001762         0.000000   \n",
       "1714   200146         0.990160         25.0         0.009840         0.000000   \n",
       "1713   199026         0.990207         29.0         0.009793         0.000000   \n",
       "1709   196830         0.981306         36.0         0.018694         0.000000   \n",
       "1707   196384         0.983809         32.0         0.016191         0.000000   \n",
       "1700   194867         0.963333         12.0         0.036667         0.000000   \n",
       "1699   194766         0.975424         60.0         0.024576         0.000000   \n",
       "1697   194634         0.987829         41.0         0.012171         0.000000   \n",
       "1696   194633         0.982690         29.0         0.017310         0.000000   \n",
       "1692   194097         0.975205         36.0         0.024795         0.000000   \n",
       "1688   192525         0.982586         63.0         0.017414         0.000000   \n",
       "1681   191104         0.982146         32.0         0.017854         0.000000   \n",
       "1670   187088         0.990946         31.0         0.009054         0.000000   \n",
       "1669   186716         0.989697         11.0         0.010303         0.000000   \n",
       "1667   186282         0.968713         14.0         0.031287         0.000000   \n",
       "1664   185057         0.971470         31.0         0.028530         0.000000   \n",
       "1659   184540         0.989574         71.0         0.010426         0.000000   \n",
       "1658   184538         0.991623         46.0         0.008377         0.000000   \n",
       "1654   184427         0.964500         18.0         0.035500         0.000000   \n",
       "1653   184404         0.987467         20.0         0.012533         0.000000   \n",
       "1652   184403         0.986272         27.0         0.013728         0.000000   \n",
       "1650   184375         0.976997         83.0         0.023003         0.000000   \n",
       "1648   184355         0.984859         59.0         0.015141         0.000000   \n",
       "1646   184274         0.978667         14.0         0.021333         0.000000   \n",
       "1641   184070         0.976638         21.0         0.023362         0.000000   \n",
       "1633   179854         0.989791         77.0         0.010209         0.000000   \n",
       "3615  2351897         0.983939         11.0         0.016061         0.000000   \n",
       "\n",
       "      sum_link_label  predicted_label         id  actual  a_sum     k  \n",
       "3595       13.000000         1.000000  2104351.0     1.0    1.0     1  \n",
       "3282       17.666667         1.000000   955631.0     0.0    1.0     2  \n",
       "2289       12.066667         1.000000   352725.0     1.0    2.0     3  \n",
       "3488       20.600000         1.000000  1455795.0     1.0    3.0     4  \n",
       "293        13.200000         1.000000    17643.0     0.0    3.0     5  \n",
       "2874        9.533333         1.000000   615155.0     0.0    3.0     6  \n",
       "62         11.000000         0.866667     5594.0     0.0    3.0     7  \n",
       "3564       10.133333         0.933333  1886070.0     0.0    3.0     8  \n",
       "3485       16.333333         1.000000  1444212.0     1.0    4.0     9  \n",
       "1956       14.466667         1.000000   262232.0     1.0    5.0    10  \n",
       "1916       14.600000         1.000000   248965.0     0.0    5.0    11  \n",
       "25         14.866667         1.000000      456.0     0.0    5.0    12  \n",
       "1770       10.466667         1.000000   215624.0     1.0    6.0    13  \n",
       "2666       10.400000         0.866667   496941.0     1.0    7.0    14  \n",
       "1665       16.466667         1.000000   185732.0     0.0    7.0    15  \n",
       "435        29.866667         1.000000    23739.0     1.0    8.0    16  \n",
       "2665       10.800000         0.800000   496919.0     1.0    9.0    17  \n",
       "449        16.866667         1.000000    23907.0     1.0   10.0    18  \n",
       "617        14.933333         0.733333    42133.0     0.0   10.0    19  \n",
       "2485        6.466667         1.000000   416873.0     0.0   10.0    20  \n",
       "3223        9.400000         1.000000   878437.0     0.0   10.0    21  \n",
       "3230        9.333333         0.800000   880719.0     1.0   11.0    22  \n",
       "2664        6.933333         0.533333   496918.0     1.0   12.0    23  \n",
       "3031        6.933333         0.733333   727279.0     0.0   12.0    24  \n",
       "2805        9.800000         0.733333   568345.0     0.0   12.0    25  \n",
       "1126       54.733333         0.733333   113506.0     0.0   12.0    26  \n",
       "2947       25.000000         0.600000   668723.0     0.0   12.0    27  \n",
       "103         6.066667         0.933333     6402.0     0.0   12.0    28  \n",
       "1769        6.533333         0.733333   215410.0     1.0   13.0    29  \n",
       "3502       17.333333         0.600000  1560662.0     0.0   13.0    30  \n",
       "...              ...              ...        ...     ...    ...   ...  \n",
       "1718        0.000000         0.000000   200549.0     0.0  226.0  3587  \n",
       "1717        0.000000         0.000000   200465.0     0.0  226.0  3588  \n",
       "1716        0.000000         0.000000   200270.0     0.0  226.0  3589  \n",
       "1715        0.000000         0.000000   200194.0     0.0  226.0  3590  \n",
       "1714        0.000000         0.000000   200146.0     0.0  226.0  3591  \n",
       "1713        0.000000         0.000000   199026.0     0.0  226.0  3592  \n",
       "1709        0.000000         0.000000   196830.0     0.0  226.0  3593  \n",
       "1707        0.000000         0.000000   196384.0     0.0  226.0  3594  \n",
       "1700        0.000000         0.000000   194867.0     0.0  226.0  3595  \n",
       "1699        0.000000         0.000000   194766.0     0.0  226.0  3596  \n",
       "1697        0.000000         0.000000   194634.0     0.0  226.0  3597  \n",
       "1696        0.000000         0.000000   194633.0     0.0  226.0  3598  \n",
       "1692        0.000000         0.000000   194097.0     0.0  226.0  3599  \n",
       "1688        0.000000         0.000000   192525.0     0.0  226.0  3600  \n",
       "1681        0.000000         0.000000   191104.0     0.0  226.0  3601  \n",
       "1670        0.000000         0.000000   187088.0     0.0  226.0  3602  \n",
       "1669        0.000000         0.000000   186716.0     1.0  227.0  3603  \n",
       "1667        0.000000         0.000000   186282.0     0.0  227.0  3604  \n",
       "1664        0.000000         0.000000   185057.0     0.0  227.0  3605  \n",
       "1659        0.000000         0.000000   184540.0     0.0  227.0  3606  \n",
       "1658        0.000000         0.000000   184538.0     0.0  227.0  3607  \n",
       "1654        0.000000         0.000000   184427.0     0.0  227.0  3608  \n",
       "1653        0.000000         0.000000   184404.0     0.0  227.0  3609  \n",
       "1652        0.000000         0.000000   184403.0     1.0  228.0  3610  \n",
       "1650        0.000000         0.000000   184375.0     0.0  228.0  3611  \n",
       "1648        0.000000         0.000000   184355.0     0.0  228.0  3612  \n",
       "1646        0.000000         0.000000   184274.0     0.0  228.0  3613  \n",
       "1641        0.000000         0.000000   184070.0     0.0  228.0  3614  \n",
       "1633        0.000000         0.000000   179854.0     0.0  228.0  3615  \n",
       "3615        0.000000         0.000000  2351897.0     0.0  228.0  3616  \n",
       "\n",
       "[3616 rows x 11 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will added additional column taht actualy will store the p@k values by calculating the precision at k."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"p@k\"] = df.apply(lambda x: x[\"actual_sum\"]/x[\"k\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>k</th>\n",
       "      <th>p@k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3595</th>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3282</th>\n",
       "      <td>2</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>3</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3488</th>\n",
       "      <td>4</td>\n",
       "      <td>0.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>293</th>\n",
       "      <td>5</td>\n",
       "      <td>0.600000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2874</th>\n",
       "      <td>6</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>7</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3564</th>\n",
       "      <td>8</td>\n",
       "      <td>0.375000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3485</th>\n",
       "      <td>9</td>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1956</th>\n",
       "      <td>10</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1916</th>\n",
       "      <td>11</td>\n",
       "      <td>0.454545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>12</td>\n",
       "      <td>0.416667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1770</th>\n",
       "      <td>13</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2666</th>\n",
       "      <td>14</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1665</th>\n",
       "      <td>15</td>\n",
       "      <td>0.466667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>16</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2665</th>\n",
       "      <td>17</td>\n",
       "      <td>0.529412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>18</td>\n",
       "      <td>0.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>19</td>\n",
       "      <td>0.526316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2485</th>\n",
       "      <td>20</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3223</th>\n",
       "      <td>21</td>\n",
       "      <td>0.476190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>22</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2664</th>\n",
       "      <td>23</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3031</th>\n",
       "      <td>24</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2805</th>\n",
       "      <td>25</td>\n",
       "      <td>0.480000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1126</th>\n",
       "      <td>26</td>\n",
       "      <td>0.461538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2947</th>\n",
       "      <td>27</td>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>28</td>\n",
       "      <td>0.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1769</th>\n",
       "      <td>29</td>\n",
       "      <td>0.448276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3502</th>\n",
       "      <td>30</td>\n",
       "      <td>0.433333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1718</th>\n",
       "      <td>3587</td>\n",
       "      <td>0.063005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1717</th>\n",
       "      <td>3588</td>\n",
       "      <td>0.062988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1716</th>\n",
       "      <td>3589</td>\n",
       "      <td>0.062970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1715</th>\n",
       "      <td>3590</td>\n",
       "      <td>0.062953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1714</th>\n",
       "      <td>3591</td>\n",
       "      <td>0.062935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1713</th>\n",
       "      <td>3592</td>\n",
       "      <td>0.062918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1709</th>\n",
       "      <td>3593</td>\n",
       "      <td>0.062900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1707</th>\n",
       "      <td>3594</td>\n",
       "      <td>0.062883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1700</th>\n",
       "      <td>3595</td>\n",
       "      <td>0.062865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1699</th>\n",
       "      <td>3596</td>\n",
       "      <td>0.062848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1697</th>\n",
       "      <td>3597</td>\n",
       "      <td>0.062830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1696</th>\n",
       "      <td>3598</td>\n",
       "      <td>0.062813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1692</th>\n",
       "      <td>3599</td>\n",
       "      <td>0.062795</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1688</th>\n",
       "      <td>3600</td>\n",
       "      <td>0.062778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1681</th>\n",
       "      <td>3601</td>\n",
       "      <td>0.062760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1670</th>\n",
       "      <td>3602</td>\n",
       "      <td>0.062743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1669</th>\n",
       "      <td>3603</td>\n",
       "      <td>0.063003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1667</th>\n",
       "      <td>3604</td>\n",
       "      <td>0.062986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>3605</td>\n",
       "      <td>0.062968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1659</th>\n",
       "      <td>3606</td>\n",
       "      <td>0.062951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1658</th>\n",
       "      <td>3607</td>\n",
       "      <td>0.062933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1654</th>\n",
       "      <td>3608</td>\n",
       "      <td>0.062916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1653</th>\n",
       "      <td>3609</td>\n",
       "      <td>0.062898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1652</th>\n",
       "      <td>3610</td>\n",
       "      <td>0.063158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1650</th>\n",
       "      <td>3611</td>\n",
       "      <td>0.063140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1648</th>\n",
       "      <td>3612</td>\n",
       "      <td>0.063123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1646</th>\n",
       "      <td>3613</td>\n",
       "      <td>0.063105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1641</th>\n",
       "      <td>3614</td>\n",
       "      <td>0.063088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1633</th>\n",
       "      <td>3615</td>\n",
       "      <td>0.063071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3615</th>\n",
       "      <td>3616</td>\n",
       "      <td>0.063053</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3616 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         k       p@k\n",
       "3595     1  1.000000\n",
       "3282     2  0.500000\n",
       "2289     3  0.666667\n",
       "3488     4  0.750000\n",
       "293      5  0.600000\n",
       "2874     6  0.500000\n",
       "62       7  0.428571\n",
       "3564     8  0.375000\n",
       "3485     9  0.444444\n",
       "1956    10  0.500000\n",
       "1916    11  0.454545\n",
       "25      12  0.416667\n",
       "1770    13  0.461538\n",
       "2666    14  0.500000\n",
       "1665    15  0.466667\n",
       "435     16  0.500000\n",
       "2665    17  0.529412\n",
       "449     18  0.555556\n",
       "617     19  0.526316\n",
       "2485    20  0.500000\n",
       "3223    21  0.476190\n",
       "3230    22  0.500000\n",
       "2664    23  0.521739\n",
       "3031    24  0.500000\n",
       "2805    25  0.480000\n",
       "1126    26  0.461538\n",
       "2947    27  0.444444\n",
       "103     28  0.428571\n",
       "1769    29  0.448276\n",
       "3502    30  0.433333\n",
       "...    ...       ...\n",
       "1718  3587  0.063005\n",
       "1717  3588  0.062988\n",
       "1716  3589  0.062970\n",
       "1715  3590  0.062953\n",
       "1714  3591  0.062935\n",
       "1713  3592  0.062918\n",
       "1709  3593  0.062900\n",
       "1707  3594  0.062883\n",
       "1700  3595  0.062865\n",
       "1699  3596  0.062848\n",
       "1697  3597  0.062830\n",
       "1696  3598  0.062813\n",
       "1692  3599  0.062795\n",
       "1688  3600  0.062778\n",
       "1681  3601  0.062760\n",
       "1670  3602  0.062743\n",
       "1669  3603  0.063003\n",
       "1667  3604  0.062986\n",
       "1664  3605  0.062968\n",
       "1659  3606  0.062951\n",
       "1658  3607  0.062933\n",
       "1654  3608  0.062916\n",
       "1653  3609  0.062898\n",
       "1652  3610  0.063158\n",
       "1650  3611  0.063140\n",
       "1648  3612  0.063123\n",
       "1646  3613  0.063105\n",
       "1641  3614  0.063088\n",
       "1633  3615  0.063071\n",
       "3615  3616  0.063053\n",
       "\n",
       "[3616 rows x 2 columns]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\"k\", \"p@k\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see in the p@k plot that the resuts are musch better then random which about 6%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0xdbe69b0>]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0xd34dba8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8HPV9//HXZ7Wr+z4sy5aNfMgX\nYGxwDNRc4QiGcPyaQAo0hUISmlCS9pfklwdp8ksCv7S0aUmgJWlKQoAchUJKAqEQ4hBCCIeNDObw\nhU/JsmzrPlfS6vj+/tiRLNuyJNvS7krzfj4e+9iZ2e/OfDUP+z3f/c7Md8w5h4iI+EMg3hUQEZHY\nUeiLiPiIQl9ExEcU+iIiPqLQFxHxEYW+iIiPKPRFRHxEoS8i4iMKfRERHwnGa8OFhYWurKwsXpsX\nEZmU1q9fX++cKzre78ct9MvKyqioqIjX5kVEJiUzqzyR76t7R0TERxT6IiI+otAXEfGRuPXpi4gc\nr56eHqqrq+nq6op3VSZMamoqpaWlhEKhcV2vQl9EJp3q6mqysrIoKyvDzOJdnXHnnKOhoYHq6mrm\nzJkzrusetXvHzH5kZrVm9t5RPjcz+1cz225m75jZ6eNaQxGRw3R1dVFQUDAlAx/AzCgoKJiQXzJj\n6dN/GFg9wueXAeXe61bg30+8WiIiI5uqgT9gov6+UUPfOfcHoHGEIlcDP3ZRrwO5ZlYy2nob2iNj\nr6WIiIyL8bh6ZyawZ8h8tbfsCGZ2q5lVmFlFbWt4HDYtIpJ4uru7+Zd/+RdWrlzJsmXLuOqqq3jl\nlVcOKVNWVkZ9fX3M6zYeoT/cb5Bhn7bunHvAObfCObciKSlpHDYtIpJYuru7ufzyy+nu7mbNmjVs\n2LCBe+65hzvvvJMnn3wy3tUbl9CvBmYNmS8FasZhvSIiCWv37t0sWrSIm266iaVLl3LNNdcQDoe5\n++67ufbaa/nKV75CTk4OAOXl5Tz11FPcc889dHZ2HrKezs5OVq9ezQ9+8IOY1Hs8Ltl8GrjdzB4D\nzgRanHP7xmG9IiKjuvNXG9lU0zqu61wyI5uvX3nyqOW2bt3Kgw8+yKpVq7jlllv43ve+x7PPPsva\ntWvZvn07N998M4FAgNNOO43rrruOa6+9lueee46PfOQjALS3t3Pddddx4403cuONN47r33A0Y7lk\n81HgNWChmVWb2SfM7NNm9mmvyLPATmA78APgtrFseNj+HxGRSWTWrFmsWrUKgI9//OO8+OKLzJo1\nCzPjjjvu4L777uPXv/41a9euJRKJsHDhQnbs2DH4/auvvpqbb745ZoEPY2jpO+euH+VzB/z1MW9Z\nqS8i42AsLfKJcvhlleFwmIyMDAAaGho4/fTobUsXXHABALW1tUybNm2w/KpVq3juuee44YYbYnYJ\nqsbeERE5TlVVVbz22msAPProo1xxxRVUVVXR19dHXl4eGzZsoKuri5deeonm5mYeeeQRrrjiisHv\n33XXXRQUFHDbbWPqIBkXcQt9NfRFZLJbvHgxjzzyCEuXLqWxsZHPfOYzXHjhhdx///3cfffd3H77\n7axevZqzzz6b73//+3zrW9+ioKDgkHXce++9dHV18aUvfSkmddbYOyIixykQCPD973//kGVf/epX\nWb16NaFQiN/+9rekpqZSVVXFmjVrWLFixWC53bt3D04/9NBDsaqyundERMZTeno6zz//PI2NjZx3\n3nmcccYZfP7znx/s3483i56Hjb3MmQtd+96tcdm2iExumzdvZvHixfGuxoQb7u80s/XOuRVH+cqo\n4tinr159ETl+8WqwxspE/X3q3hGRSSc1NZWGhoYpG/wD4+mnpqaO+7p1IldEJp3S0lKqq6upq6uL\nd1UmzMCTs8abQl9EJp1QKDTuT5TyC12nLyLiI+rTFxHxkfiFvpr6IiIxp+4dEREfUfeOiIiPKPRF\nRHxEd+SKiPiITuSKiPiIundERHxEoS8i4iO6ZFNExEfU0hcR8RGFvoiIj8Q19KfqWNgiIokqrqHf\nr8wXEYkptfRFRHxELX0RER+Jc+gr9UVEYklX74iI+Iha+iIiPqI+fRERH9HVOyIiPqKWvoiIj8T3\nRK5CX0QkpsYU+ma22sy2mtl2M7tjmM9nm9mLZvaWmb1jZpePZb06kSsiElujhr6ZJQHfBS4DlgDX\nm9mSw4p9FXjcObccuA743lg2rtAXEYmtsbT0VwLbnXM7nXMR4DHg6sPKOCDbm84BasaycUW+iEhs\njSX0ZwJ7hsxXe8uG+gbwcTOrBp4FPjvciszsVjOrMLMKUEtfRCTWxhL6Nsyyw9P6euBh51wpcDnw\nEzM7Yt3OuQeccyuccyui88daXRERORFjCf1qYNaQ+VKO7L75BPA4gHPuNSAVKBxtxQp9EZHYGkvo\nvwGUm9kcM0smeqL26cPKVAEXAZjZYqKhXzfaitW9IyISW6OGvnOuF7gdeB7YTPQqnY1mdpeZXeUV\n+wLwKTN7G3gU+Es3htttFfkiIrFl8RoKIaWk3G3f+Daz8tPjsn0RkcnIzNYPnBc9HnEeeyeeWxcR\n8Z/4hr46eEREYkoDromI+IgeoiIi4iNxDf2//5/N3PSjdfGsgoiIrwTjufHfbamN5+ZFRHxHD0YX\nEfERhb6IiI8o9EVEfEShLyLiIwp9EREfUeiLiPhIQoR+vAZ9ExHxm4QI/V6NxyAiEhMJEfo9ff3x\nroKIiC8kSOirpS8iEgsJEvpq6YuIxEJChH6vWvoiIjGREKGvlr6ISGwo9EVEfCQhQl+XbIqIxEZC\nhH6kVy19EZFYSIjQV0tfRCQ2EiL01acvIhIbCn0RER9JkNBX946ISCwkROj3qqUvIhITCRH66t4R\nEYmNBAl9de+IiMRCQoR+b79a+iIisZAQod/Tq5a+iEgsJEboq6UvIhITiRH6GoZBRCQmxhT6Zrba\nzLaa2XYzu+MoZT5mZpvMbKOZ/eexVELDMIiIxEZwtAJmlgR8F7gEqAbeMLOnnXObhpQpB74MrHLO\nNZnZtGOpRESXbIqIxMRYWvorge3OuZ3OuQjwGHD1YWU+BXzXOdcE4JyrPZZK6MlZIiKxMZbQnwns\nGTJf7S0bagGwwMxeMbPXzWz1cCsys1vNrMLMKgYrYBpaWUQkVsYS+jbMssOb5kGgHLgAuB74oZnl\nHvEl5x5wzq1wzq0YWJYSTFL3johIjIwl9KuBWUPmS4GaYco85Zzrcc7tArYSPQiMKiUUoLunbyxF\nRUTkBI0l9N8Ays1sjpklA9cBTx9W5pfABwHMrJBod8/OsVQgJRigW907IiIxMWroO+d6gduB54HN\nwOPOuY1mdpeZXeUVex5oMLNNwIvA/3HONYylAinBJLrU0hcRiYlRL9kEcM49Czx72LKvDZl2wOe9\n1zFRS19EJHbifkduSkihLyISK3EP/dRgEt296t4REYmFuId+9OodtfRFRGIh/qEfTKJrhJb+W1VN\nlN3xP2zZ3xrDWomITE0JEPojt/R/9fY+AF7cUherKomITFmJEfojnMjtdxqXR0RkvMQ99FNDI5/I\n7ejuBaC+vTtWVRIRmbLiHvqjtfT3NncCUOO9i4jI8Yt/6IeSRuzTHwz9lq5YVUlEZMqKf+gHA3T1\n9uGG6bt3zrGvORr2aumLiJy4hAh956BnmAepNId7iPT1k5seor69m0hvP7f9bD2PrauKQ01FRCa/\nBAj9JIBhT+bub4228pfNysU5qGrs4Ln39rNm04GY1lFEZKqIe+inhqJVGO5k7oEhoQ/w+s5GnIOd\n9R2xq6CIyBQS99A/2NI/MvRrW6OXaS6fnQfA6zujozVXNYb1iEURkeMQ/9D3WvpdPX3UtXWzo64d\ngM37Wnn/QBsAy0oHWvrR0O/rd1Q1huNQWxGRyW1M4+lPlBvPPomU4MHQ//C/vkxXTz+77r6cy+57\nGYC89BA56SHy0kPUt0cGv7ujrp350zLjUm8Rkckqbi39U2bk8I0rTyY9OXrc6Yz00eVdr9855Ela\nxdmpAMzOTwegrCD6vrNO/foiIscqbqFvBoGAkZ4c7dMPRw4GfV3bwSEXpg2EfkEGAKfMzKEwM4Wd\nXjeQiIiMXVy7dwDSBkO/d3DZ7oaD/fXFWSkAzMiJhn9pXjpzi7p1BY+IyHGI+4ncDK97Z2hLf9eQ\nVvxA904gYAAkBwPMK8pkR137sHfxiojI0cU99Ae6dzqGhP7Qln52WvSgcG55IQCr5hWwsDiT5nAP\ntW0aeVNE5FjEvXsnPcVr6Xcf7N4Z2nUTsGgL/0/mFbLxzkvJSAky0L7fvK918JeAiIiMLu4t/bRQ\ntKU/dLz83UNCf0VZ/uB0hneAWDQ9C4At+9tiUUURkSkj7i39pICRGgpwoPVg6Fc1hslJC/HqHRcO\nBv1QuenJlOSksmWfnpsrInIs4t7SB0hPDg6OszMgPyN52MAfsGh61mBLv6evn5Zwz4TWUURkKkiQ\n0E864qRsXnpoxO8sKslme207v99ay5d+/g4Xf+clevs0Ho+IyEji3r0D0dCvbjr0ISl56ckjfmfR\n9Cx6+x1/+dAbg8u27G/jlJk5E1JHEZGpICFa+mnJwcHr9HPSoi38vIyRQ39JSfYRy9ZXNo1/5URE\nppCECP0M71p9ODjGTv4ooT+36MjB1t7Y3Ti+FRMRmWISIvTTh4R+kTfsQu4offpJ3h26Awozk6nY\n3aS7dEVERpAgoX/w1MLAFTsD3TwjufK0GQC8/fUP8bmLytnf2sVePUBdROSoEiL0s1KjQR8wSAuN\nvUrf+dhpbLzzUnLSQqw4KXoT19qd6uIRETmaBAn9aKs+PTlIZkp0OhQYvWrBpMAhd+nmZyTz3Hv7\n+IsH1x5yV6+IiESNKfTNbLWZbTWz7WZ2xwjlrjEzZ2YrjqUSA4OqJQWMv7m4nL86by5XL59xLKsg\nEDDOnlfAbzfX8vK2en6+vvqYvi8i4gejhr6ZJQHfBS4DlgDXm9mSYcplAZ8D1h5rJQZa+n39jpy0\nEF++fPHgA9OPxap5hYPTL2ypPebvi4hMdWNp6a8EtjvndjrnIsBjwNXDlPt/wLeArmE+G1G216ff\n239id9Suml8wOL15X6tO6oqIHGYsoT8T2DNkvtpbNsjMlgOznHPPjLQiM7vVzCrMrKKurm5w+cCJ\n3BPMfGbnp7OgOHMw/H+n1r6IyCHGEvo2zLLBi+HNLAB8B/jCaCtyzj3gnFvhnFtRVFQ0uDx7oHvn\nBK+xNzN+cdsqHvrLlZQVpPPC5gMntD4RkalmLKFfDcwaMl8K1AyZzwJOAX5vZruBs4Cnj+Vk7tA+\n/ROVkRIkORjgwkXFvLqj4ZBn74qI+N1YQv8NoNzM5phZMnAd8PTAh865FudcoXOuzDlXBrwOXOWc\nqxhrJQa6d8bTRYunEent5w/v14/7ukVEJqtRQ9851wvcDjwPbAYed85tNLO7zOyq8ahE9hjuvj1W\nK+fkk5+RzK/eqRm9sIiIT4ypie2cexZ49rBlXztK2QuOtRIDA64NPAZxPISSAlx+6nR+vr6a9u5e\nMkd4IIuIiF8kxB25ZsbPP302P/vkmeO63quXzaSrp581m/aP63pFRCarhGn+Dn0A+ng5Y3YeM3JS\neXpDDZ2RfvIzkll9yvRx346IyGSRMKE/EQIB48rTZvDDP+7ixa11TMtK4aLF0wglJcQPHBGRmJvy\n6XfVshmDl4LWtnWzZpOu3RcR/5ryoT/0sYqZKUF++nplHGsjIhJfUz70zYx/u345Z87J59Pnz+XV\nHQ1sr22Pd7VEROJiSvfpD7jytBlcedoM6tu7ue+Fbfz09UrmT8skIyWJP11eGu/qiYjEjC9Cf0Bh\nZgpXLJ3B4xV7SAoY/f2OCxZMI2+Uh7CLiEwVU75753C3njeXcKSPtq5eOiJ9PPjHXfGukohIzPgu\n9BeXZHPegugIn8tm5fLwq7tpDkfiXCsRkdjwXegDfP3KJdxx2SL+8aOn0t7dyw9fVmtfRPzBl6E/\nryiTT58/j0XTs/nw0hIe/OMuDrQe8wO/REQmHV+G/lBfunQhvf393PObrfGuiojIhPN96J9UkMFN\nZ5fxxPpqNu9rjXd1REQmlO9DH+CzF5aTkxbirl9twp3gIxtFRBKZQh/ISQ/xxQ8t5LWdDfxyw954\nV0dEZMIo9D03rJzNslm5fPOZzbSEe+JdHRGRCaHQ9wQCxj/86ak0d/Zw93ObAdhU00pnpI8vPvE2\n63Y1Dpatbgrzjac36uAgIpOOr4ZhGM2SGdl88tw5/MdLOzm1NIf/+8v3WFySzcaaVn63pZZnPnsO\nnT19/N2T77J2VyNb97fxyC0rSQ7q2Ckik4PF68TlihUrXEVFRVy2PZKunj6uvv8Vth5oG1wWDBip\noSTmT8vk/QNthCN9ZKUEaevu5WMrSvmnjy7FzOJYaxHxCzNb75xbcbzfVxP1MKmhJO752GkEA0ZJ\nTirnlhdyyZJi/vmapWzY00w40gfAAzeu4HMXlfN4RTXf+/0OALYdaKO9uzee1RcRGZG6d4Zxyswc\n7rtuOUkB40NLijGLjst/63lzefCPu3jxCxcwuyCds+bmU9nQwT8/H72x697fvs+8okz+81Nnka+R\nO0UkAal75xg459jf2kVJTtrgskhvP7f+pILfb60DIDkpwNyijCOCv997ZOOW/W1s2tfKR0+fqS4h\nETlm6t6JITM7JPABkoMB/v3Pz+Cc+YWcOSefH960gl31Hdzwg9epb+8eLHf3c5s591svcvujb/LF\nJ97ms4++RafXVSQiEitq6Y+jvn5HUsB4eVsdn/pxBdOzU3n45pV87rG3eKe6ZbDcn8wr4LWdDZwy\nI4cHbjzjiAOJiMjRnGhLX6E/QdZXNnLLwxX09PUPnvz9wY0r2Fbbxs1/ModXd9TzuUffIj0lyP3X\nL+fMuQVxrrGITAYK/QT2/oE2bnxwHb39jhe/eD5ZqaEjPv+rn6ynsqGDz1+ygNsumE8goH5+ETk6\nhX6Cawn30B7pZWbu8F047d29fOUX7/LUhhrOLS/k2x9bRlFWSoxrKSKThU7kJric9NBRAx8gMyXI\nvX+2jH/8yKms29XIpff+gV+/t5/H39jD1d99hdd3NsSwtiIy1amln0C2HWjjfz++gff2tpKcFCDS\n1w/A9Stnccdli8lJC42yBhGZ6tTSn0LKi7P4xW2r+NxF5SQHAzx88we49by5/Ncbe7j42y/x7Lv7\nNN6/iJwQtfQTlHNu8Oatd6tbuOPJd9hY08qq+QV87YqTaQpHeHRdFZ88Zy6nlubEubYiEis6kesT\nvX39/GxtFd9e8z7t3b1kpwZp8oZ2vvK0GXzxQws4qSAD5xxN4R4NAyEyRcUk9M1sNXAfkAT80Dn3\nj4d9/nngk0AvUAfc4pyrHGmdCv3j09QR4dtr3ufRdVX8w0dOZU9jmB++vIuevn5uOHM2AD95vZLL\nTpnO7R8sZ8mM7DjXWETG04SHvpklAe8DlwDVwBvA9c65TUPKfBBY65wLm9lngAucc3820noV+icm\n0ts/OI5/bWsX972wjcfe2ENfv2NBcSY1zV20d/dy8eJp3H5hOctm5ca5xiIyHmIR+mcD33DOXerN\nfxnAOXf3UcovB+53zq0aab0K/fG3q76Dn6/fw8fPOon0UJBHXtvNg3/cRUtnD6vmF/DJc+eSkhTg\nW89v5YqlJVx7xixy0nVFkMhkEovQvwZY7Zz7pDf/F8CZzrnbj1L+fmC/c+6bw3x2K3ArwOzZs8+o\nrByxB0jGQXt3Lz99vZKHXtnFgdZukgJGKMno6uknNRTgT5fP5C/OKlM3kMgkcaKhP5bx9IcbF2DY\nI4WZfRxYAZw/3OfOuQeAByDa0h9jHeUEZKYE+fT587hl1RyeeaeGJ9/cy+0XzicrNchPXqvkF2/t\n5dF1e/hAWR7Xr5zNZaeUkJacFO9qi8gEGbfuHTO7GPg34HznXO1oG1b3TmJoDkd4oqKan66tpLIh\nTGZKMNr1s6KU02fnacx/kQQTi+6dINETuRcBe4meyL3BObdxSJnlwM+JdgNtG8uGFfqJpb/fsW53\nI09UVPPsu/vo7OljblEGHz29lCuWlnBSQUa8qygixO6SzcuBe4lesvkj59zfm9ldQIVz7mkz+y1w\nKrDP+0qVc+6qkdap0E9c7d29PPvOPp5Yv4c3djcBsLQ0hyuWlvDhpTOOOpaQcw7n0EihIhNIN2fJ\nhKpuCvPsu/t45p19gw+COX12LpefWsKlJ09nVn4622vbeLOymYrKRp55Zx/nzC/k4sXFXLCoiLRQ\nEv/x0k7KizM5Z34hBZkaQVTkRCj0JWYqGzp45p3oAWDzvlYAFpdk0xyOsK+lC4CVZflUN4Wp8eaL\nslKoa4s+NtIMTpmRw7nlhZy3oIjTZ+cN3mtwuEhvP3/cXsePX6vktNJczpyTz/LZeTrJLL6n0Je4\n2F3fwZpNB/jNpv28t7eVf/jIKfT3R4eECCUZm/e18bstB3jp/TouPXk6HyjL5w/v1/GHbXW8WdVM\nX78jIzmJs+cVcs78AuYUZfLNZzYxMy+Ns+cW8N9vVvP+gXayU4O0d/fS7yAYME4tzWHlnHzOnJPP\nGSfla+RR8R2Fvkw6bV09vLqjYfAgsKexE4CctBCFmcnsqOsAogeQv7t8ERkpQdZXNvHGrkbW7Wrk\n7epmevocZrBoejbLZ+eybFYup8/OZW5hps4pyJSm0JdJb09jmNd3NrC4JJtTZuZQ29ZFa2cP86dl\nDVu+q6ePt6qaWberkYrKRjbsaaatqxeArNQgp5VGDwIDB4OCzBR213ewYU8zi0uymVeUQTDp2EcV\n7+7t47l395ObHqK+PcLzG/czpzCDBcVZLCzOorw4k9SQup9kYin0xff6+x0769t5q6qZDXuiry37\n2+jrj/7bLs1LoyXcQ1t39MCQEgywaHoWJ8/M4eQZ2ZwyI4eF07MOCeyfra3kuXf3M39aJoumZ7Go\nJJt/e2EbL2w5eAtKUVYKLZ09RHqjD7sxg7KCDBYUZ7KwOIsF07NYND2LkwoyCB3HQUZkOAp9kWF0\nRvp4d28LG/Y08XZ1C21dvXzm/HkcaO3ivb0tbKxpZWNNC63eL4SkgFE+LZMlM7LJTAny49cqmZ2f\nTkN7Nx2RvsH1fv3KJSwuyaaurZtLlhQTDBi7G8K8f6CNrfvbou8H2thd34F3zCE5KUBZYTrzijKZ\nW5TB3ELvvShT5yTkmCn0RY6Tc47qpk421kQPAgMHg1ov0L/356eTZMaepjCb97WRkxbi7HkFY1p3\nV08fO+raef9AG1v2t7Gjtp2ddR1UNoYHf4EAFGamMLcog3lDDgbzijIpzUsbsQuqtq2L/n6YlpVC\nIGBsqmklHOllVn46RZkpOq8xhSn0RcZZW1cPWakT0wLv6eunqjEcPQjUd7CzLnow2FnfQWNHZLBc\nMGCU5qVxUkEGZQXpzC7IYGZuGr98ay/hnj5e3V5Pb78jJRhgZl4aO72T3wDJwQCleWnMyktnVn4a\npXnpg9Oz8tLJTQ9peI1JLBYDron4ykQFPkAoKcC8okzmFWUe8VlTR4Sd9e3sqOtgd30HlQ1hKhs7\nWF/ZRLt3PiI9OYnZ+elcevJ0zppXQFVDtNwHF07jnPJCqhvD7GnqZE9jmD1NYTbsaaals+eQ7WSm\nBKMHhfx0SvPSmJGTRkluKjNyo9NFWSkk6ZfClKXQF0kQeRnJnJERvf9gKOccjR0RdjeEKcmJhvOx\naO3qobqxkz1NYfY0hqn2DgpVDWFe3V5/yDkLiP7KKM5OZUZuKiU5adGDgTf9VlUTu+o7KM5OpTg7\nlek5KRRnpVKck8r07FQyUhQpiU7dOyI+5pyjtauXmuZO9rV0UtPc5U1H32taOtnf0kVP38GcmFOY\nQV1b9+Cvj6EyU4IUZ6dEDwjZqUzLTmW6N1+cEz1QFGWmHPVO7BP5O5rDPTR0RFi3q5Hc9BAFGckU\nZCZTkJFCTlpoypznUPeOiBw3MyMnLUROWojFJcM/SKe/31Hf3k1NSxcpwcBgufbuXg60dg15dbO/\npYvati72t3SxdlcjtW2HHjAG5KWHKMxMoSjLew2ZHrq8prmTNyubyM9MoTAzefCA8ZuNB0gNBSjI\nTOH1nQ28vK2eXfUdR2xnQFLAyEtPpjAzmfyMZAoyU6IHBW86PyP62cB0dmpwyp73UOiLyIgCAWOa\n12ofKjMlSOZRzk8M6O93NIUj7G/tora1m/2tXdS1dR98tXfzVlUzdW3ddPb0HXU9I9bPomNAffFD\nC0gKBLhkSTGR3n4aOyI0dHRT3x6hsaObhvYIDR0RGtq7ebe6mYb2yOC9G4cLJVn04JARPQjkpoe8\n92Ty00PkZSSTl+69MkLkpSezt7mTzftayU4LkZsWIjc9mdy0ENlpoYQ6R6LQF5EJEwhYtFWdmcLJ\nM45ezjlHR6SPeu9AMHBQ6Onr57JTS+jo7qW+PRrgTR0RzppbQF5GiIb2CNlpoaMO9z2a7t6+6MFh\nyAGhsSNyyIGiMRxhb3MnjR2RI06Kj1V2apCc9BC5adEDSE5aiFxvPict5H3mHSi86ey00ITc4a3Q\nF5G4M7PoL4eUIGWFwz+wZ0HxkcNyTMtKHabk2KUEkyjJSaMkZ2wHjb5+R0tnD40dEZrDEe+9h8Zw\nBAPOKS+ku7eflnAPzZ3Rz5rDPbR0Rl/N4QjNnT3sbeqk2ZvvH+G0amoocPDAkBYiO+3EI1uhLyIy\nRkmBaLdPfkbyuKzPOUd7d+/ggaF5yMHikANFuIfWrh72Nned8DYV+iIicWJmZKWGyEoNMWus3/nb\nE9umRoESEfERhb6IiI8o9EVEfEShLyLiIwp9EREfUeiLiPhI/C7ZrN8GD304bpsXEfEjtfRFRHwk\nfi39wnK4+X/itnkRkUnplhMbvE0tfRERH1Hoi4j4iEJfRMRHFPoiIj6i0BcR8RGFvoiIjyj0RUR8\nRKEvIuIj5twID2icyA2b1QGVcdl4YikE6uNdiQSi/XGQ9sVB2hcHLXTOHfnA4DGK2x25zrmieG07\nkZhZhXNuRbzrkSi0Pw7SvjhI++IgM6s4ke+re0dExEcU+iIiPqLQj78H4l2BBKP9cZD2xUHaFwed\n0L6I24lcERGJPbX0RUR8RKECfbXdAAAEAUlEQVQ/wczsR2ZWa2bvDVmWb2ZrzGyb957nLTcz+1cz\n225m75jZ6fGr+fgzs1lm9qKZbTazjWb2N95y3+0PM0s1s3Vm9ra3L+70ls8xs7XevvgvM0v2lqd4\n89u9z8viWf+JYGZJZvaWmT3jzftyX5jZbjN718w2DFypM57/RxT6E+9hYPVhy+4AXnDOlQMvePMA\nlwHl3utW4N9jVMdY6QW+4JxbDJwF/LWZLcGf+6MbuNA5dxqwDFhtZmcB/wR8x9sXTcAnvPKfAJqc\nc/OB73jlppq/ATYPmffzvvigc27ZkMtUx+//iHNOrwl+AWXAe0PmtwIl3nQJsNWb/g/g+uHKTcUX\n8BRwid/3B5AOvAmcSfQGpKC3/GzgeW/6eeBsbzrolbN4130c90GpF2YXAs8A5uN9sRsoPGzZuP0f\nUUs/Poqdc/sAvPdp3vKZwJ4h5aq9ZVOO95N8ObAWn+4PrztjA1ALrAF2AM3OuV6vyNC/d3BfeJ+3\nAAWxrfGEuhf4EtDvzRfg333hgN+Y2Xozu9VbNm7/R+L3jFwZznAPv5xyl1eZWSbw38DfOudazY76\nzM8pvT+cc33AMjPLBX4BLB6umPc+ZfeFmV0B1Drn1pvZBQOLhyk65feFZ5VzrsbMpgFrzGzLCGWP\neV+opR8fB8ysBMB7r/WWVwOzhpQrBWpiXLcJZWYhooH/M+fck95i3+4PAOdcM/B7ouc5cs1soDE2\n9O8d3Bfe5zlAY2xrOmFWAVeZ2W7gMaJdPPfiz32Bc67Ge68l2hhYyTj+H1Hox8fTwE3e9E1E+7YH\nlt/onZE/C2gZ+Ek3FVi0Sf8gsNk59+0hH/luf5hZkdfCx8zSgIuJnsR8EbjGK3b4vhjYR9cAv3Ne\nJ+5k55z7snOu1DlXBlxH9G/7c3y4L8wsw8yyBqaBDwHvMZ7/R+J90mKqv4BHgX1AD9Gj8ieI9j++\nAGzz3vO9sgZ8l2jf7rvAinjXf5z3xTlEf3q+A2zwXpf7cX8AS4G3vH3xHvA1b/lcYB2wHXgCSPGW\np3rz273P58b7b5ig/XIB8Ixf94X3N7/tvTYCX/GWj9v/Ed2RKyLiI+reERHxEYW+iIiPKPRFRHxE\noS8i4iMKfRERH1Hoi3jMrMyGjIYqMhUp9EVEfEShLzIMM5vrje3+gXjXRWQ8KfRFDmNmC4mOD3Sz\nc+6NeNdHZDxplE2RQxURHdfko865jfGujMh4U0tf5FAtRMcnXxXviohMBLX0RQ4VAf4X8LyZtTvn\n/jPeFRIZTwp9kcM45zq8B3usMbMO59xTo35JZJLQKJsiIj6iPn0RER9R6IuI+IhCX0TERxT6IiI+\notAXEfERhb6IiI8o9EVEfEShLyLiI/8fhArT0id3TSMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xdbe6860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.figure()\n",
    "df[[\"k\", \"p@k\"]][:500].plot(x=\"k\", y= \"p@k\")\n",
    "plt.plot(df[[\"k\"]].values, np.full((len(df[[\"k\"]]),1), 0.06))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
